#!/usr/bin/env python

#################################################################################
#
# Copyright (c) 2006 Michigan State University
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
#################################################################################

#################################################################################
#
#	Author:
#		Nathan Collins <npcollins@gmail.com>
#
#################################################################################

"""
OAI metadata parsing.
"""

from types import *
import urllib
import httpget
from time import sleep
from oailog import *
from querymgr import *
from oaixml import *
from utility import *
from oaivars import *


def getxmltag(xmldict, tag):
	"""
	Return the first instance of tag in xmldict.
	"""

	# loop through the xmldict
	for key,val in xmldict.iteritems():
		# find tag
		if key.strip()[-len(tag):] == tag:
			return val

	# not found
	return None


def parseIdentify(server):
	getArgs = '?verb=Identify'
	doc = getXML(server + getArgs)
	xmlDict = xmlToDict(doc)
	identify = {}

	# retrieve first tier of xml data
	try:
		idenDict = xmlDict['OAI-PMH']['Identify']
	except:
		# some repositories don't feel the need for the standard root element
		idenDict = xmlDict['Identify']
	
	for key in idenDict:
		if type(idenDict[key]) is DictType and 'value' in idenDict[key]:
			identify[key] = idenDict[key]['value']

	# return server identification info
	return identify

def parseListIdentifiers(server,token='',datefrom=None,dateto=None):
	getArgs = '?verb=ListIdentifiers'
	if len(token) > 0:
		getArgs += '&resumptionToken=' + urllib.quote_plus(token)

	else:
		getArgs += '&metadataPrefix=oai_dc'
		if datefrom is not None:
			getArgs += '&from=%s' % datefrom
		if dateto is not None:
			getArgs += '&until=%s' % dateto

	doc = getXML(server + getArgs)

	# get list of identifiers and the token
	ident, tok = getList(doc)
	return ident, tok

def getList(listXML):
	"""
	Given an xml tree, pull the list of identifiers and resuption token and return.
	"""
	idList = []
	token = None
	oaipmh = getFromList( 'OAI-PMH', listXML.getChild() )
	listId = getFromList( 'ListIdentifiers', oaipmh )
	header = getFromList( 'header', listId )
	# pull out the indentifiers
	for line in header:
		if line.getName() == 'identifier':
			idList.append(line.getContent())

	# retreive token
	for line in listId:
		if line.getName() == 'resumptionToken':
			token = line.getContent()

	# return id list and token
	return idList, token

def getFromList(field, xmlList):
	list = []
	for c in xmlList:
		if c.getName() == field:
			list += c.getChild()
	return list

def parseGetRecord(server, ident):
	getArgs = '?verb=GetRecord&metadataPrefix=oai_dc&identifier='
	getArgs += ident
	doc = getXML(server + getArgs)

	xmlDict = xmlToDict(doc)
	rec = {}

	# direct link back to metadata
	rec['iMetadataDirect'] = server + getArgs

	try:
		root_elem = getxmltag(xmlDict, 'OAI-PMH')
		gr_elem = getxmltag(root_elem, 'GetRecord')
	
		# get header info
		header = getxmltag(getxmltag(gr_elem,'record'),'header')

		# check if record is deleted
		if header['attr'].has_key('status'):
			if header['attr']['status'] == 'deleted':
				rec['deleted'] = 'yes'
				return rec

		# get record info
		meta_elem = getxmltag(getxmltag(gr_elem,'record'), 'metadata')
		
		metadata = getxmltag(meta_elem, 'dc')

	except:
		try:
			# not a valid record, get error
			code = xmlDict['OAI-PMH']['error']['attr']['code']
			mssg = xmlDict['OAI-PMH']['error']['value']
			rec = { 'error':'yes', 'code':code, 'message':mssg }
			return rec

		except:
			# something new
			rec = { 'uncaught':'yes'}
			return rec

	rec = checkFields(rec,header,metadata)

	return rec

def checkFields(rec,header,metadata):
	# get fields (header)
	rec['hIdentifier'] = getRecordField(header,'identifier')
	rec['hDatestamp'] = getRecordField(header,'datestamp')
	rec['hDeleted'] = '' # future work
	# get fields (metadata)
	rec['mTitle'] = getRecordField(metadata,'title')
	rec['mCreator'] = getRecordField(metadata,'creator')
	rec['mSubject'] = getRecordField(metadata,'subject')
	rec['mDescription'] = getRecordField(metadata,'description')
	rec['mPublisher'] = getRecordField(metadata,'publisher')
	rec['mContributor'] = getRecordField(metadata,'contributor')
	rec['mDate'] = getRecordField(metadata,'date')
	rec['mType'] = getRecordField(metadata,'type')
	rec['mFormat'] = getRecordField(metadata,'format')
	rec['mIdentifier'] = getRecordField(metadata,'identifier')
	rec['mSource'] = getRecordField(metadata,'source')
	rec['mLanguage'] = getRecordField(metadata,'language')
	rec['mRelation'] = getRecordField(metadata,'relation')
	rec['mCoverage'] = getRecordField(metadata,'coverage')
	rec['mRights'] = getRecordField(metadata,'rights')
	# return the metadata dict
	return rec

def parseListRecords(server, token,datefrom=None,dateto=None):
	getArgs = '?verb=ListRecords'

	# token
	if token != '':
		getArgs += '&resumptionToken=' + token
	else:
		getArgs += '&metadataPrefix=oai_dc'
		if datefrom is not None:
			getArgs += '&from=%s' % datefrom
		if dateto is not None:
			getArgs += '&until=%s' % dateto

	# pull XML
	doc = getXML(server + getArgs)
	lr = None
	token = None

	# find ListRecords
	for c in doc.getChild()[0].getChild():
		if c.getName() == 'ListRecords':
			lr = c.getChild()

	parsedList = []

	for r in lr:
		if r.getName() == 'resumptionToken':
			token = r.getContent()
			continue

		record = xmlToDict(r)
		rec = {}
		rec['iMetadataDirect'] = server
		metadata = {}

		try:
			header = record['header']
			# check if record is deleted
			if header['attr'].has_key('status'):
				if header['attr']['status'] == 'deleted':
					rec['deleted'] = 'yes'
					parsedList.append(rec)
			# metadata
			metadata = getxmltag(getxmltag(record,'metadata'),'dc')

		except:
			try:
				# not a valid record, get error
				code = xmlDict['OAI-PMH']['error']['attr']['code']
				mssg = xmlDict['OAI-PMH']['error']['value']
				rec = { 'error':'yes', 'code':code, 'message':mssg }
				parsedList.append(rec)
				continue

			except:
				# something new
				rec = { 'uncaught':'yes'}
				parsedList.append(rec)
				continue

		rec = checkFields(rec,header,metadata)
		parsedList.append(rec)

	return parsedList, token


def getRecordField(recDict, field):
	if recDict.has_key(field):
		return recDict[field]['value']
	elif recDict.has_key('dc:' + field):
		return recDict['dc:' + field]['value']
	return ''


class harvestRepository:
	def __init__(self, server,datefrom=None,dateto=None):
		# datefrom and dateto are for selective harvesting
		# server url
		self.server = server
		# identity dict
		self.identity = None
		# list identifiers
		self.identifiers = []
		# resumption token
		self.token = None
		# selective harvesting dates
		self.datefrom = datefrom
		self.dateto = dateto
		# getRecord works, will switch to listRecords if it doesn't
		self.useGetRecord = g['USEGR']
		# recordList is for holding record when getRecord fails
		self.recordList = []
		# identify server
		self.identify()
		self.getList()

	def identify(self):
		try:
			self.identity = parseIdentify(self.server)
		except:
			# Could not identify server
			swrite('I')
			hErr('Failed to identify server: %s' % self.server)
			raise

	def getList(self,token = None):
		if token is None:
			# first list retrieval
			iList, tok = parseListIdentifiers(self.server,'',self.datefrom,self.dateto)
		else:
			# use token
			iList, tok = parseListIdentifiers(self.server,token)

		self.identifiers = iList

		# token will be None is there is no token
		self.token = tok

	def getRecord(self):
		record = None

		if self.useGetRecord == 1:
			if len(self.identifiers) > 0:
				# pull a record from the list
				record = self.pullRecord()

			elif self.token is not None and self.token.strip() != '':
				# there is a resumption token
				self.getList(self.token)
				record = self.pullRecord()

		else:
			# using ListRecords
			if len(self.recordList) > 0:
				record = self.addServerInfo(self.recordList.pop())

			elif self.token is not None and self.token.strip() != '':
				# get next ListRecords if possible
				self.recordList, self.token = parseListRecords(self.server,self.token)
				if len(self.recordList) > 0:
					record = self.addServerInfo(self.recordList.pop())

		return record

	def pullRecord(self):
		try:
			# get one record and return it
			# there are identifiers to pull
			ident = self.identifiers.pop()
			record = parseGetRecord(self.server, ident)

			if record.has_key('error'):
				if record['code'] == 'badArgument': raise
		except:
			# getRecord may not work on this repository, try listRecords
			swrite('R')
			hErr("GetRecord failed. Trying ListRecords. - %s" % self.server)
			self.useGetRecord = 0

			self.recordList, self.token = parseListRecords(self.server, '')

			if len(self.recordList) > 0:
				return self.addServerInfo(self.recordList.pop())
			else:
				return None

		if record.has_key('error'):
			# check for second try
			if record['code'].strip() == '503':
				sleep(35)	# wait and retry
				record = parseGetRecord(self.server, ident)

			# if error persists, record and skip
			if record.has_key('error'):
				swrite('O')
				#print 'OAIError - Code: %s   Message: %s' % (record['code'],record['message'])
				hErr('InvalidXML: %s - Record: %s  OAIErrorCode: %s >> %s' % \
					(self.server,ident,record['code'],record['message']))
				return record
		return self.addServerInfo(record)

	def addServerInfo(self, record):
		# add server info to record dict, doing some renaming
		record = self.tryCopy(self.identity,'earliestDatestamp',record,'iDatestamp')
		record = self.tryCopy(self.identity,'repositoryName',record,'iRepository')
		record = self.tryCopy(self.identity,'baseURL',record,'iOAIURL')
		record = self.tryCopy(self.identity,'protocolVersion',record,'iProtocolVer')
		record = self.tryCopy(self.identity,'deletedRecord',record,'iHandleDelete')
		record = self.tryCopy(self.identity,'adminEmail',record,'iContact')
		return record

	def tryCopy(self,olddict,oldkey,newdict,newkey):
		"""
		attempt to copy the old key from the old dict to the 
		new dict using the new key. if no old key exists,
		the newdict has newkey as an empty string
		"""
		newdict[newkey] = ''
		try:
			newdict[newkey] = olddict[oldkey]
		except:
			pass
		return newdict

def recreateTable():
	# query to delete the old table
	dropQuery = "DROP TABLE IF EXISTS harvestSearch;"
	dbquery(dropQuery)
	initTable()

def initTable():
	# query to recreate the table
	createQuery = """
		CREATE TABLE harvestSearch (
		recHash VARCHAR(32) NOT NULL PRIMARY KEY,
		hDatestamp VARCHAR(32) NOT NULL,
		iDatestamp VARCHAR(32) NOT NULL,
		hDeleted VARCHAR(16) NOT NULL,
		iRepository TEXT NOT NULL,
		iContact TEXT NOT NULL,
		hIdentifier TEXT NOT NULL,
		mTitle TEXT NOT NULL,
		mCreator TEXT NOT NULL,
		mSubject TEXT NOT NULL,
		mDescription TEXT NOT NULL,
		mPublisher TEXT NOT NULL,
		mContributor TEXT NOT NULL,
		mDate TEXT NOT NULL,
		mType TEXT NOT NULL,
		mFormat TEXT NOT NULL,
		mIdentifier TEXT NOT NULL,
		mSource TEXT NOT NULL,
		mLanguage TEXT NOT NULL,
		mRelation TEXT NOT NULL,
		mCoverage TEXT NOT NULL,
		mRights TEXT NOT NULL,
		iOAIURL TEXT NOT NULL,
		iHandleDelete VARCHAR(16) NOT NULL,
		iProtocolVer VARCHAR(16) NOT NULL,
		iMetadataDirect TEXT NOT NULL,
		cDups INT DEFAULT 0,
		cLanguage TEXT NOT NULL,
		cValidUrl BOOLEAN NOT NULL DEFAULT 1,
		cTimestamp TIMESTAMP
		)
		CHARACTER SET utf8
		MAX_ROWS = 200000000000
		AVG_ROW_LENGTH = 800;
		"""

	dbquery(createQuery)

# eof

